package lpg;

import java.util.ArrayList;
import java.util.List;

class Scanner
{
    Scanner(Option option, LexStream lexStream,
            LookupTable<String, VariableSymbol> variableTable, LookupTable<String, MacroSymbol> macroTable)
    {
        this.option         = option;
        this.lexStream      = lexStream;
        this.variableTable  = variableTable;
        this.macroTable     = macroTable;
        this.actionBlocks   = option.actionBlocks;
    }

    private void reportErrors()
    {
        String msg = null;
        for (int i = 0, n = warningTokens.size(); i < n; i++)
        {
            ProblemToken warningToken = warningTokens.get(i);
            Token token = warningToken.token;
            String name = warningToken.name;
            assert (name != null);
            switch (warningToken.msgCode)
            {
                case LEGACY_KEYWORD:
                    String name_ = name.substring(1);
                    msg = "The use of the escaped symbol \"" + option.escape + name_ +
                            "\" as a keyword has been deprecated. The proper spelling is %" + name_;
                    break;
                case SYMBOL_WITH_KEYWORD_MARKER:
                    msg = "The symbol \"" + name + "\" starts with the keyword marker %. It should be quoted.";
                    break;
                case INCLUDE_OPTIONS:
                    msg = "Some options may have been imported from the included file \"" + name + "\"";
                    break;
                default:
                    assert false;
            }

            option.emitWarning(token, msg);
        }

        int numErrorTokens = errorTokens.size();
        if (numErrorTokens > 0)
        {
            for (int i = 0; i < numErrorTokens; i++)
            {
                ProblemToken errorToken = errorTokens.get(i);
                Token token = errorToken.token;
                String name = errorToken.name;
                switch (errorToken.msgCode)
                {
                    case NO_INPUT:
                        msg = "No input file specified";
                        break;
                    case NO_TEMPLATE:
                        msg = "The template file " + option.templateName + " could not be read.";
                        break;
                    case NO_INCLUDE:
                        msg = "The include file " + name + " could not be read.";
                        break;
                    case RECURSIVE_INCLUDE:
                        msg = "Attempt to recursively include file " + name;
                        break;
                    case BAD_UNICODE:
                        msg = "Symbol \"" + name + "\" contains an invalid unicode character.";
                        break;
                    case BAD_OCTAL_ASCII_CODE:
                        msg = "Symbol \"" + name + "\" contains an invalid ASCII octal code.";
                        break;
                    case ISOLATED_BACKSLASH:
                        msg = "Symbol \"" + name + "\" contains an isolated backslash.";
                        break;
                    case UNDELIMITED_STRING_SYMBOL:
                        msg = "Quoted symbol not properly delimited. The closing quote must be followed by a space";
                        break;
                    case UNTERMINATED_STRING_SYMBOL:
                        msg = "Quoted symbol not properly terminated";
                        break;
                    case UNTERMINATED_BRACKET_SYMBOL:
                        msg = "Symbol enclosed in angular brackets not properly terminated";
                        break;
                    case UNTERMINATED_BLOCK:
                        msg = "Block not properly terminated";
                        break;
                    default:
                        assert false;
                }
                option.emitError(token, msg);
            }
            throw new LpgExitException();
        }
    }

    //
    // Scan the main file
    //
    void scan()
    {
        lexStream.resetTokenStream();

        inputFile = lexStream.findOrInsertFile(option.grmFile);
        if (inputFile == null)
        {
            option.report.append("***Error: Input file \"");
            option.report.append(option.grmFile);
            option.report.append("\" could not be read");
            option.flushReport(false);
        }
        else
        {
            inputFile.lock();
            inputFile.readInput();
            if (inputFile.buffer() == null)
            {
                option.report.append("***Error: Input file \"");
                option.report.append(option.grmFile);
                option.report.append("\" could not be read");
                option.flushReport(false);
                return;
            }

            currentTokenIndex = lexStream.getNextToken(inputFile, 0); // Get 0th token and set its location to 0!
            currentToken = lexStream.getTokenReference(currentTokenIndex);
            currentToken.setKind(0);

            //
            // Assign the null string symbol to the 0th token.
            //
            currentToken.setSymbol(variableTable.findOrInsert(
                    "", new VariableSymbol("", variableTable.size())));
            currentToken.setEndLocation(0);

            //
            // Process all options if any that was specified by the user or set
            // defaults otherwise. First, we process the options that were hard-wired
            // in the input. Then, we process any option that was issued from the
            // command line.
            //
            int mainCursor = scanOptions();

            option.processCommandOptions(); // process options from command line

            //
            // If we need to include a template, load its environment and
            // process its options.
            //
            InputFileSymbol templateFile = null;
            int templateCursor = 0;
            if (option.templateName != null)
            {
                //
                // First, look in the template search directory path for the
                // template file name.
                //
                templateFile = lexStream.findOrInsertFile(option.templateSearchDirectory, option.templateName);

                //
                // If we were not successful in the template search directory path,
                // look in the include search directory path.
                //
                if (templateFile == null)
                    templateFile = lexStream.findOrInsertFile(option.includeSearchDirectory, option.templateName);

                //
                // If we could not open the template file, issue an error.
                //
                if (templateFile == null)
                    addErrorToken(StreamErrorKind.NO_TEMPLATE, currentTokenIndex);
                else
                {
                    templateFile.lock();
                    templateFile.readInput();
                    if (templateFile.buffer() == null)
                    {
                        option.emitError(0, "Input file \"" + option.templateName + "\" could not be read");

                        return;
                    }
                    pushEnvironment();
                    inputFile = templateFile;
                    templateCursor = scanOptions(); // scan the template options
                }
            }

            option.completeOptionProcessing();
            if (option.errorFlag) // if any bad option was found, stop
            {
                option.flushReport(false);
                throw new LpgExitException();
            }
            option.printOptionsInEffect();

            //
            // If a template is present, continue scanning it...
            //
            if (templateFile != null)
            {
                scan(templateCursor);
                while (lexStream.kind(currentTokenIndex) == Lpgsym.TK_EOF)
                    currentTokenIndex--;
                lexStream.resetTokenStream(currentTokenIndex + 1);
                templateFile.unlock(); // unlock the template file.
                popEnvironment();
            }

            //
            // Restore the main file environment and continue scanning
            //
            scan(mainCursor);

            for (int k = 0, n = option.importFile.size(); k < n; k++)
                importTerminalsOrProcessFilters(option.importFile.get(k),
                        option.symFile, option.prefix, option.suffix, true);

            for (int k = 0, n = option.filterFile.size(); k < n; k++)
                importTerminalsOrProcessFilters(option.filterFile.get(k),
                        option.expFile, option.expPrefix, option.expSuffix, false);
        }

        //
        // If any problem tokens were scanned, flag them.
        //
        reportErrors();
    }

    //
    // Scan the file specified
    //
    void scanFile(int fileIndex)
    {
        inputFile = lexStream.findOrInsertFile(option.includeSearchDirectory, lexStream.nameString(fileIndex));

        StreamErrorKind errorCode = includeFile();
        if (errorCode != StreamErrorKind.NO_ERROR)
            addErrorToken(errorCode, fileIndex);
        currentTokenIndex = lexStream.getNextToken(inputFile, cursor);
        currentToken = lexStream.getTokenReference(currentTokenIndex);
        currentToken.setKind(Lpgsym.TK_EOF);

        //
        // If any problem tokens were scanned, flag them.
        //
        reportErrors();
    }

    private int scanOptions()
    {
        inputBuffer = inputFile.buffer();
        inputEnd = inputBuffer.length;
        cursor = 0;
        lineLocation = inputFile.lineLocation();
        lineLocation.clear();
        lineLocation.add(0); // mark starting location of line # 0

        while (true)
        {
            skipSpaces();
            if (cursor == inputEnd || inputBuffer[cursor] != '%')
                break;

            int startCursor = cursor;
            if (Util.prefixMatch(inputBuffer, cursor + 1, inputEnd, "INCLUDE", "include") == 7)
            {
                cursor += 8;
                skipSpaces();
                if (cursor == inputEnd)
                {
                    resetEnvironment(startCursor);
                    return startCursor; // we need to rescan this INCLUDE file
                }
                int ptr = cursor + 1;
                char c;
                while (ptr < inputEnd && !Character.isWhitespace(c = inputBuffer[ptr]) && c != option.escape)
                    ptr++;
                String filename = new String(inputBuffer, cursor, ptr);
                InputFileSymbol includeFile = lexStream.findOrInsertFile(option.includeSearchDirectory, filename);
                if (includeFile == null || includeFile.isLocked())
                {
                    resetEnvironment(startCursor);
                    return startCursor; // we need to rescan this INCLUDE file
                }

                Token errorToken = lexStream.getErrorToken(inputFile, cursor);
                errorToken.setEndLocation(ptr - 1);
                errorToken.setKind(0);
                addWarningToken(StreamErrorKind.INCLUDE_OPTIONS, includeFile.name(), errorToken);

                includeFile.readInput();
                if (includeFile.buffer() == null)
                {
                    resetEnvironment(startCursor);
                    return startCursor;
                }
                cursor = ptr; // we will need to continue from where we left off
                pushEnvironment();
                includeFile.lock();
                inputFile = includeFile;

                int returnPtr = scanOptions();

                includeFile.unlock(); // unlock the include file.
                popEnvironment();
                if (returnPtr < includeFile.buffer().length) // did not reach end of include file?
                {
                    resetEnvironment(startCursor);
                    return startCursor;
                }
                skipSpaces();
                if (cursor < inputEnd && inputBuffer[cursor] == '%' &&
                        Util.prefixMatch(inputBuffer, cursor + 1, inputEnd, "END", "end") == 3)
                    cursor += 4;
            }
            else if (Util.prefixMatch(inputBuffer, cursor + 1, inputEnd, "OPTIONS", "options") == 7)
            {
                int start = cursor + 8;
                char c = inputBuffer[start];
                if (!Character.isWhitespace(c) && !(c == '-' && inputBuffer[start + 1] == '-'))
                    break;
                //
                // Note that we start scanning at "start" in case start points to a
                // new line character or a comment.
                //
                int ptr = start;
                while ((c = inputBuffer[ptr]) != '\r' && c != '\n' && !(c == '-' && inputBuffer[ptr + 1] == '-'))
                    ptr++; // skip all until \n
                option.processUserOptions(inputFile, inputBuffer, start, ptr);

                cursor = ptr;
            }
            else
                break;
        }

        return cursor;
    }

    private void skipOptions()
    {
        for (skipSpaces();
             cursor < inputEnd && inputBuffer[cursor] == '%' &&
                    Util.prefixMatch(inputBuffer, cursor + 1, inputEnd, "OPTIONS", "options") == 7;
             skipSpaces())
        {
            int start = cursor + 8;
            char c = inputBuffer[start];
            if (!Character.isWhitespace(c) && !(c == '-' && inputBuffer[start + 1] == '-'))
                break;
            //
            // Note that we start scanning at "start" in case start points to a
            // new line character or a comment.
            //
            int ptr = start;
            while ((c = inputBuffer[ptr]) != '\r' && c != '\n' && !(c == '-' && inputBuffer[ptr + 1] == '-'))
                ptr++; // skip all until \n
            cursor = ptr;
        }
    }

    private void importTerminalsOrProcessFilters(String filename,
            String expFile, String expPrefix, String expSuffix, boolean isImportTerminals)
    {
        InputFileSymbol fileSymbol = lexStream.findOrInsertFile(option.includeSearchDirectory, filename);
        String name = (fileSymbol == null ? filename : fileSymbol.name());
        Arguments arguments = new Arguments(option, expFile, expPrefix, expSuffix, name);
        Option option = new Option(arguments.args);
        LexStream lexStream = new LexStream(option);
        LookupTable<String, VariableSymbol> variableTable = new LookupTable<String, VariableSymbol>();
        LookupTable<String, MacroSymbol> macroTable = new LookupTable<String, MacroSymbol>();
        Scanner scanner = new Scanner(option, lexStream, variableTable, macroTable);
        scanner.scan();

        if (lexStream.numTokens() == 0 || scanner.numErrorTokens() > 0)
            throw new LpgExitException();

        //
        // This is the code one would write in order to retrieve only the
        // terminals exported from the imported file without processing the
        // whole grammar.
        //
        //     Parser parser = new Parser(option, lexStream, variableTable, macroTable);
        //     parser.parse();
        //
        //     //
        //     // Transfer each imported terminal symbol into the local variable
        //     // table and add it to the list of imported symbols.
        //     //
        //     IntArrayList imports = new IntArrayList();
        //     for (int i = 0, n = lexStream.numImportedFilters(); i < n; i++)
        //         imports.add(lexStream.importedFilter(i));
        //     imports.addAll(parser.exports);
        //
        //     for (int i = 0, n = imports.size(); i < n; i++)
        //     {
        //         int imp = imports.get(i);
        //
        //         VariableSymbol symbol = lexStream.getVariableSymbol(imp);
        //         InputFileSymbol oldFileSymbol = lexStream.getFileSymbol(imp);
        //         InputFileSymbol newFileSymbol = this.lexStream.findOrInsertFile(oldFileSymbol.name());
        //         if (newFileSymbol.lineLocation().isEmpty())
        //             newFileSymbol.lineLocation().addAll(oldFileSymbol.lineLocation());
        //
        //         currentTokenIndex = this.lexStream.getNextToken(newFileSymbol, lexStream.startLocation(imp));
        //         currentToken = this.lexStream.getTokenReference(currentTokenIndex);
        //         currentToken.setEndLocation(lexStream.endLocation(imp));
        //         currentToken.setKind(lexStream.kind(imp));
        //         String variableName = symbol.name();
        //         currentToken.setSymbol(this.variableTable.findOrInsert(
        //             variableName, new VariableSymbol(variableName, this.variableTable.size())));
        //
        //         if (isImportTerminals)
        //             this.lexStream.addImportedTerminal(currentTokenIndex);
        //         else
        //             this.lexStream.addImportedFilter(currentTokenIndex);
        //     }
        //

        //
        // Process the imported grammar.
        //
        Control control = new Control(option, lexStream, variableTable, macroTable);
        control.process();

        //
        // Merge the symbols inherited from filter files
        // with the symbols that are to be exported by the parser.
        //
        IntArrayList imports = new IntArrayList();
        for (int i = 0, n = lexStream.numImportedFilters(); i < n; i++)
            imports.add(lexStream.importedFilter(i));
        imports.addAll(control.grammar.parser.exports);

        //
        // Contruct the final list of imported symbols in lex_stream.
        //
        for (int i = 0, n = imports.size(); i < n; i++)
        {
            int imp = imports.get(i);

            VariableSymbol symbol = lexStream.getVariableSymbol(imp);
            InputFileSymbol oldFileSymbol = lexStream.getFileSymbol(imp);
            InputFileSymbol newFileSymbol = this.lexStream.findOrInsertFile(oldFileSymbol.name());
            if (newFileSymbol.lineLocation().isEmpty()) // first encounter of this file?
                newFileSymbol.lineLocation().addAll(oldFileSymbol.lineLocation());

            currentTokenIndex = this.lexStream.getNextToken(newFileSymbol, lexStream.startLocation(imp));
            currentToken = this.lexStream.getTokenReference(currentTokenIndex);
            currentToken.setEndLocation(lexStream.endLocation(imp));
            currentToken.setKind(lexStream.kind(imp));
            String variableName = symbol.name();
            currentToken.setSymbol(this.variableTable.findOrInsert(
                    variableName, new VariableSymbol(variableName, this.variableTable.size())));

            if (isImportTerminals)
                this.lexStream.addImportedTerminal(currentTokenIndex);
            else
                this.lexStream.addImportedFilter(currentTokenIndex);
        }
    }

    //
    // This is one of the main entry point for the lexical analyser.
    // Its input is the name of a regular text file. Its output is a stream
    // of tokens.
    //
    private void scan(int bufferStart)
    {
        cursor = bufferStart;

        //
        // CURSOR is assumed to point to the next character to be scanned.
        // Using CURSOR,we jump to the proper classification function
        // which scans and classifies the token and returns the location of
        // the character immediately following it.
        //
        for (skipSpaces(); cursor < inputEnd; skipSpaces())
            classifyToken();

        // Add eof
        currentTokenIndex = lexStream.getNextToken(inputFile, cursor);
        currentToken = lexStream.getTokenReference(currentTokenIndex);
        currentToken.setKind(Lpgsym.TK_EOF);

        //
        // Add a gate after the last line.
        //
        lineLocation.add(inputEnd); //Phileas TODO: see skipSpaces, is this necessary?
    }

    private void scanComment()
    {
        char c;
        for (cursor += 2; (c = inputBuffer[cursor]) != '\r' && c != '\n'; cursor++)  // skip all until \n
            ;
    }

    //
    // This procedure is invoked to skip useless spaces in the input.
    // It assumes upon entry that CURSOR points to the next character to
    // be scanned.  Before returning it sets CURSOR to the location of the
    // first non-space character following its initial position.
    //
    private void skipSpaces()
    {
        while (cursor < inputEnd)
        {
            char c = inputBuffer[cursor];
            if (Character.isWhitespace(c))
            {
                cursor++;
                if (c == '\n') // For the sequence \r\n only count \n.
                    lineLocation.add(cursor);
            }
            else if (c == '-' && inputBuffer[cursor + 1] == '-')
                scanComment();
            else
                break;
        }
    }

    //
    // Scan a symbol of length I and determine if it is a keyword.
    //
    private int scanKeyword0(int p1, int len)
    {
        switch (len)
        {
            case 4:
                return scanKeyword4(p1);
            case 6:
                return scanKeyword6(p1);
            case 7:
                return scanKeyword7(p1);
            case 8:
                return scanKeyword8(p1);
            case 9:
                return scanKeyword9(p1);
            case 10:
                return scanKeyword10(p1);
            case 11:
                return scanKeyword11(p1);
            case 12:
                return scanKeyword12(p1);
            case 13:
                return scanKeyword13(p1);
            case 24:
                return scanKeyword24(p1);
            default:
                return Lpgsym.TK_SYMBOL;
        }
    }

    //
    // Scan a symbol of length I and determine if it is a keyword.
    //
    private int scanKeyword4(int p1)
    {
        char c;
        if ((c = inputBuffer[p1 + 1]) == 'e' || c == 'E')
        {
            if ((c = inputBuffer[p1 + 2]) == 'o' || c == 'O')
            {
                 if ((c = inputBuffer[p1 + 3]) == 'l' || c == 'L')
                      return Lpgsym.TK_EOL_KEY;
                 else if ((c = inputBuffer[p1 + 3]) == 'f' || c == 'F')
                      return Lpgsym.TK_EOF_KEY;
            }
            else if (((c = inputBuffer[p1 + 2]) == 'n' || c == 'N') &&
                     ((c = inputBuffer[p1 + 3]) == 'd' || c == 'D'))
                 return Lpgsym.TK_END_KEY;
        }
        else if (((c = inputBuffer[p1 + 1]) == 'a' || c == 'A') &&
                 ((c = inputBuffer[p1 + 2]) == 's' || c == 'S') &&
                 ((c = inputBuffer[p1 + 3]) == 't' || c == 'T'))
             return Lpgsym.TK_AST_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword6(int p1)
    {
        char c;
        switch (inputBuffer[p1 + 1])
        {
            case 'a': case 'A':
                if (((c = inputBuffer[p1 + 2]) == 'l' || c == 'L') &&
                    ((c = inputBuffer[p1 + 3]) == 'i' || c == 'I') &&
                    ((c = inputBuffer[p1 + 4]) == 'a' || c == 'A') &&
                    ((c = inputBuffer[p1 + 5]) == 's' || c == 'S'))
                    return Lpgsym.TK_ALIAS_KEY;
                break;
            case 'e': case 'E':
                if (((c = inputBuffer[p1 + 2]) == 'm' || c == 'M') &&
                    ((c = inputBuffer[p1 + 3]) == 'p' || c == 'P') &&
                    ((c = inputBuffer[p1 + 4]) == 't' || c == 'T') &&
                    ((c = inputBuffer[p1 + 5]) == 'y' || c == 'Y'))
                    return Lpgsym.TK_EMPTY_KEY;
                else if (((c = inputBuffer[p1 + 2]) == 'r' || c == 'R') &&
                         ((c = inputBuffer[p1 + 3]) == 'r' || c == 'R') &&
                         ((c = inputBuffer[p1 + 4]) == 'o' || c == 'O') &&
                         ((c = inputBuffer[p1 + 5]) == 'r' || c == 'R'))
                    return Lpgsym.TK_ERROR_KEY;
                break;
            case 'n': case 'N':
                if (((c = inputBuffer[p1 + 2]) == 'a' || c == 'A') &&
                    ((c = inputBuffer[p1 + 3]) == 'm' || c == 'M') &&
                    ((c = inputBuffer[p1 + 4]) == 'e' || c == 'E') &&
                    ((c = inputBuffer[p1 + 5]) == 's' || c == 'S'))
                    return Lpgsym.TK_NAMES_KEY;
                break;
            case 'r': case 'R':
                if (((c = inputBuffer[p1 + 2]) == 'u' || c == 'U') &&
                    ((c = inputBuffer[p1 + 3]) == 'l' || c == 'L') &&
                    ((c = inputBuffer[p1 + 4]) == 'e' || c == 'E') &&
                    ((c = inputBuffer[p1 + 5]) == 's' || c == 'S'))
                    return Lpgsym.TK_RULES_KEY;
                break;
            case 's': case 'S':
                if (((c = inputBuffer[p1 + 2]) == 't' || c == 'T') &&
                    ((c = inputBuffer[p1 + 3]) == 'a' || c == 'A') &&
                    ((c = inputBuffer[p1 + 4]) == 'r' || c == 'R') &&
                    ((c = inputBuffer[p1 + 5]) == 't' || c == 'T'))
                    return Lpgsym.TK_START_KEY;
                break;
            case 't': case 'T':
                if (((c = inputBuffer[p1 + 2]) == 'y' || c == 'Y') &&
                    ((c = inputBuffer[p1 + 3]) == 'p' || c == 'P') &&
                    ((c = inputBuffer[p1 + 4]) == 'e' || c == 'E') &&
                    ((c = inputBuffer[p1 + 5]) == 's' || c == 'S'))
                    return Lpgsym.TK_TYPES_KEY;
                break;
        }

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword7(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 2]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 3]) == 'f' || c == 'F') &&
            ((c = inputBuffer[p1 + 4]) == 'i' || c == 'I') &&
            ((c = inputBuffer[p1 + 5]) == 'n' || c == 'N') &&
            ((c = inputBuffer[p1 + 6]) == 'e' || c == 'E'))
            return Lpgsym.TK_DEFINE_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 'e' || c == 'E') &&
                 ((c = inputBuffer[p1 + 2]) == 'x' || c == 'X') &&
                 ((c = inputBuffer[p1 + 3]) == 'p' || c == 'P') &&
                 ((c = inputBuffer[p1 + 4]) == 'o' || c == 'O') &&
                 ((c = inputBuffer[p1 + 5]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 6]) == 't' || c == 'T'))
            return Lpgsym.TK_EXPORT_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 'i' || c == 'I') &&
                 ((c = inputBuffer[p1 + 2]) == 'm' || c == 'M') &&
                 ((c = inputBuffer[p1 + 3]) == 'p' || c == 'P') &&
                 ((c = inputBuffer[p1 + 4]) == 'o' || c == 'O') &&
                 ((c = inputBuffer[p1 + 5]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 6]) == 't' || c == 'T'))
            return Lpgsym.TK_IMPORT_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 'n' || c == 'N') &&
                 ((c = inputBuffer[p1 + 2]) == 'o' || c == 'O') &&
                 ((c = inputBuffer[p1 + 3]) == 't' || c == 'T') &&
                 ((c = inputBuffer[p1 + 4]) == 'i' || c == 'I') &&
                 ((c = inputBuffer[p1 + 5]) == 'c' || c == 'C') &&
                 ((c = inputBuffer[p1 + 6]) == 'e' || c == 'E'))
            return Lpgsym.TK_NOTICE_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword8(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 'g' || c == 'G') &&
            ((c = inputBuffer[p1 + 2]) == 'l' || c == 'L') &&
            ((c = inputBuffer[p1 + 3]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 4]) == 'b' || c == 'B') &&
            ((c = inputBuffer[p1 + 5]) == 'a' || c == 'A') &&
            ((c = inputBuffer[p1 + 6]) == 'l' || c == 'L') &&
            ((c = inputBuffer[p1 + 7]) == 's' || c == 'S'))
            return Lpgsym.TK_GLOBALS_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 'h' || c == 'H') &&
                 ((c = inputBuffer[p1 + 2]) == 'e' || c == 'E') &&
                 ((c = inputBuffer[p1 + 3]) == 'a' || c == 'A') &&
                 ((c = inputBuffer[p1 + 4]) == 'd' || c == 'D') &&
                 ((c = inputBuffer[p1 + 5]) == 'e' || c == 'E') &&
                 ((c = inputBuffer[p1 + 6]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 7]) == 's' || c == 'S'))
            return Lpgsym.TK_HEADERS_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 'i' || c == 'I') &&
                 ((c = inputBuffer[p1 + 2]) == 'n' || c == 'N') &&
                 ((c = inputBuffer[p1 + 3]) == 'c' || c == 'C') &&
                 ((c = inputBuffer[p1 + 4]) == 'l' || c == 'L') &&
                 ((c = inputBuffer[p1 + 5]) == 'u' || c == 'U') &&
                 ((c = inputBuffer[p1 + 6]) == 'd' || c == 'D') &&
                 ((c = inputBuffer[p1 + 7]) == 'e' || c == 'E'))
            return Lpgsym.TK_INCLUDE_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 2]) == 'e' || c == 'E') &&
                 ((c = inputBuffer[p1 + 3]) == 'c' || c == 'C') &&
                 ((c = inputBuffer[p1 + 4]) == 'o' || c == 'O') &&
                 ((c = inputBuffer[p1 + 5]) == 'v' || c == 'V') &&
                 ((c = inputBuffer[p1 + 6]) == 'e' || c == 'E') &&
                 ((c = inputBuffer[p1 + 7]) == 'r' || c == 'R'))
            return Lpgsym.TK_RECOVER_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword9(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 'k' || c == 'K') &&
            ((c = inputBuffer[p1 + 2]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 3]) == 'y' || c == 'Y') &&
            ((c = inputBuffer[p1 + 4]) == 'w' || c == 'W') &&
            ((c = inputBuffer[p1 + 5]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 6]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 7]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 8]) == 's' || c == 'S'))
            return Lpgsym.TK_SOFTKEYWORDS_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 't' || c == 'T') &&
                 ((c = inputBuffer[p1 + 2]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 3]) == 'a' || c == 'A') &&
                 ((c = inputBuffer[p1 + 4]) == 'i' || c == 'I') &&
                 ((c = inputBuffer[p1 + 5]) == 'l' || c == 'L') &&
                 ((c = inputBuffer[p1 + 6]) == 'e' || c == 'E') &&
                 ((c = inputBuffer[p1 + 7]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 8]) == 's' || c == 'S'))
            return Lpgsym.TK_TRAILERS_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword10(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 2]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 3]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 4]) == 'p' || c == 'P') &&
            ((c = inputBuffer[p1 + 5]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 6]) == 'u' || c == 'U') &&
            ((c = inputBuffer[p1 + 7]) == 'l' || c == 'L') &&
            ((c = inputBuffer[p1 + 8]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 9]) == 's' || c == 'S'))
            return Lpgsym.TK_DROPRULES_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 't' || c == 'T') &&
                 ((c = inputBuffer[p1 + 2]) == 'e' || c == 'E') &&
                 ((c = inputBuffer[p1 + 3]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 4]) == 'm' || c == 'M') &&
                 ((c = inputBuffer[p1 + 5]) == 'i' || c == 'I') &&
                 ((c = inputBuffer[p1 + 6]) == 'n' || c == 'N') &&
                 ((c = inputBuffer[p1 + 7]) == 'a' || c == 'A') &&
                 ((c = inputBuffer[p1 + 8]) == 'l' || c == 'L') &&
                 ((c = inputBuffer[p1 + 9]) == 's' || c == 'S'))
            return Lpgsym.TK_TERMINALS_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword11(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 'i' || c == 'I') &&
            ((c = inputBuffer[p1 + 2]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 3]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 4]) == 'n' || c == 'N') &&
            ((c = inputBuffer[p1 + 5]) == 't' || c == 'T') &&
            ((c = inputBuffer[p1 + 6]) == 'i' || c == 'I') &&
            ((c = inputBuffer[p1 + 7]) == 'f' || c == 'F') &&
            ((c = inputBuffer[p1 + 8]) == 'i' || c == 'I') &&
            ((c = inputBuffer[p1 + 9]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 10]) == 'r' || c == 'R'))
            return Lpgsym.TK_IDENTIFIER_KEY;
        if (((c = inputBuffer[p1 + 1]) == 'p' || c == 'P') &&
            ((c = inputBuffer[p1 + 2]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 3]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 4]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 5]) == 't' || c == 'T') &&
            ((c = inputBuffer[p1 + 6]) == 'h' || c == 'H') &&
            ((c = inputBuffer[p1 + 7]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 8]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 9]) == 'e' || c == 'E' || c == 'i' || c == 'I') &&
            ((c = inputBuffer[p1 + 10]) == 's' || c == 'S'))
            return Lpgsym.TK_PROSTHESES_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword12(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 2]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 3]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 4]) == 'p' || c == 'P') &&
            ((c = inputBuffer[p1 + 5]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 6]) == 'y' || c == 'Y') &&
            ((c = inputBuffer[p1 + 7]) == 'm' || c == 'M') &&
            ((c = inputBuffer[p1 + 8]) == 'b' || c == 'B') &&
            ((c = inputBuffer[p1 + 9]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 10]) == 'l' || c == 'L') &&
            ((c = inputBuffer[p1 + 11]) == 's' || c == 'S'))
            return Lpgsym.TK_DROPSYMBOLS_KEY;
        else if (((c = inputBuffer[p1 + 1]) == 'd' || c == 'D') &&
                 ((c = inputBuffer[p1 + 2]) == 'r' || c == 'R') &&
                 ((c = inputBuffer[p1 + 3]) == 'o' || c == 'O') &&
                 ((c = inputBuffer[p1 + 4]) == 'p' || c == 'P') &&
                 ((c = inputBuffer[p1 + 5]) == 'a' || c == 'A') &&
                 ((c = inputBuffer[p1 + 6]) == 'c' || c == 'C') &&
                 ((c = inputBuffer[p1 + 7]) == 't' || c == 'T') &&
                 ((c = inputBuffer[p1 + 8]) == 'i' || c == 'I') &&
                 ((c = inputBuffer[p1 + 9]) == 'o' || c == 'O') &&
                 ((c = inputBuffer[p1 + 10]) == 'n' || c == 'N') &&
                 ((c = inputBuffer[p1 + 11]) == 's' || c == 'S'))
            return Lpgsym.TK_DROPACTIONS_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword13(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 2]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 3]) == 'f' || c == 'F') &&
            ((c = inputBuffer[p1 + 4]) == 't' || c == 'T') &&
            ((c = inputBuffer[p1 + 5]) == 'k' || c == 'K') &&
            ((c = inputBuffer[p1 + 6]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 7]) == 'y' || c == 'Y') &&
            ((c = inputBuffer[p1 + 8]) == 'w' || c == 'W') &&
            ((c = inputBuffer[p1 + 9]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 10]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 11]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 12]) == 's' || c == 'S'))
            return Lpgsym.TK_SOFTKEYWORDS_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private int scanKeyword24(int p1)
    {
        char c;
        if (((c = inputBuffer[p1 + 1]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 2]) == 'i' || c == 'I') &&
            ((c = inputBuffer[p1 + 3]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 4]) == 'j' || c == 'J') &&
            ((c = inputBuffer[p1 + 5]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 6]) == 'i' || c == 'I') &&
            ((c = inputBuffer[p1 + 7]) == 'n' || c == 'N') &&
            ((c = inputBuffer[p1 + 8]) == 't' || c == 'T') &&
            ((c = inputBuffer[p1 + 9]) == 'p' || c == 'P') &&
            ((c = inputBuffer[p1 + 10]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 11]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 12]) == 'd' || c == 'D') &&
            ((c = inputBuffer[p1 + 13]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 14]) == 'c' || c == 'C') &&
            ((c = inputBuffer[p1 + 15]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 16]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 17]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 18]) == 'o' || c == 'O') &&
            ((c = inputBuffer[p1 + 19]) == 'r' || c == 'R') &&
            ((c = inputBuffer[p1 + 20]) == 's' || c == 'S') &&
            ((c = inputBuffer[p1 + 21]) == 'e' || c == 'E') &&
            ((c = inputBuffer[p1 + 22]) == 't' || c == 'T') &&
            ((c = inputBuffer[p1 + 23]) == 's' || c == 'S'))
            return Lpgsym.TK_DISJOINTPREDECESSORSETS_KEY;

        return Lpgsym.TK_SYMBOL;
    }

    private void classifyToken()
    {
        //
        // Allocate space for next token and set its location.
        //
        currentTokenIndex = lexStream.getNextToken(inputFile, cursor);
        currentToken = lexStream.getTokenReference(currentTokenIndex);

        char c = inputBuffer[cursor];
        List<BlockSymbol> blocks = actionBlocks.actionBlocks(c);
        if (blocks != null)
             classifyBlock(blocks);
        else
        {
            switch (c)
            {
                case '%':
                    classifyKeyword();
                    break;
                case ':':
                    classifyEquivalence();
                    break;
                case '-':
                    classifyArrow();
                    break;
                case '\'':
                    classifySingleQuotedSymbol();
                    break;
                case '"':
                    classifyDoubleQuotedSymbol();
                    break;
                case '<':
                    classifyLess();
                    break;
                default:
                    if (c == option.orMarker)
                        classifyOr();
                    else if (c == option.escape)
                        classifyEscapedSymbol();
                    else
                        classifySymbol();
            }
        }
    }

    private void classifyBlock(List<BlockSymbol> blocks)
    {
        for (int i = 0, n = blocks.size(); i < n; i++)
        {
            BlockSymbol block = blocks.get(i);
            String blockBegin = block.blockBegin();
            String blockEnd = block.blockEnd();
            int blockBeginLength = blockBegin.length();
            int blockEndLength = blockEnd.length();
            if (Util.prefixMatch(inputBuffer, cursor, inputEnd, blockBegin) == blockBeginLength)
            {
                for (cursor = cursor + blockBeginLength;
                     Util.prefixMatch(inputBuffer, cursor, inputEnd, blockEnd) != blockEndLength;
                     cursor++)
                {
                    if (inputBuffer[cursor] == '\n') // when we encounter the sequence \r\n, we only count \n
                        lineLocation.add(cursor + 1); // character after newline.
                }

                if (cursor == inputEnd)
                    addErrorToken(StreamErrorKind.UNTERMINATED_BLOCK, currentTokenIndex);
                else
                    cursor += blockEndLength;

                currentToken.setKind(Lpgsym.TK_BLOCK);
                currentToken.setSymbol(block);
                currentToken.setEndLocation(cursor - 1);

                return;
            }
        }

        classifySymbol();
    }

    private void classifyEscapedSymbol()
    {
        int ptr = cursor + 1;

        char c;
        while (Character.isLetterOrDigit(c = inputBuffer[ptr]) && c != option.escape)
            ptr++;
        int len = ptr - cursor;
        int kind = scanKeyword0(cursor, len);
        currentToken.setEndLocation(ptr - 1);

        if (kind == Lpgsym.TK_SYMBOL || !option.legacy) // not a keyword?
        {
            currentToken.setKind(Lpgsym.TK_MACRO_NAME); // ... then it's a macro
            String macroName = new String(inputBuffer, cursor, len);
            currentToken.setSymbol(macroTable.findOrInsert(
                    macroName, new MacroSymbol(macroName, macroTable.size())));
        }
        else if (option.escape != '%') // if legacy option is on treat as keyword only
        {
            assert option.legacy;
            currentToken.setKind(kind); // it's a keyword
            addWarningToken(StreamErrorKind.LEGACY_KEYWORD, currentTokenIndex);
            if (kind == Lpgsym.TK_INCLUDE_KEY)
                ptr = processInclude(ptr);
        }

        cursor = ptr;
    }

    private void classifyKeyword()
    {
        int ptr = cursor + 1;

        char c;
        while (!Character.isWhitespace(c = inputBuffer[ptr]) && c != option.escape)
            ptr++;
        int len = ptr - cursor;

        currentToken.setKind(scanKeyword0(cursor, len));
        currentToken.setEndLocation(ptr - 1);
        if (currentToken.kind() == Lpgsym.TK_SYMBOL)
        {
            String variableName = new String(inputBuffer, cursor, len);
            currentToken.setSymbol(variableTable.findOrInsert(
                    variableName, new VariableSymbol(variableName, variableTable.size())));
            addWarningToken(StreamErrorKind.SYMBOL_WITH_KEYWORD_MARKER, currentTokenIndex);
        }
        else if (currentToken.kind() == Lpgsym.TK_INCLUDE_KEY)
             ptr = processInclude(ptr);

        cursor = ptr;
    }

    private void classifySymbol()
    {
        int ptr = cursor + 1;
        char c;
        while (!Character.isWhitespace(c = inputBuffer[ptr]) && c != option.escape)
            ptr++;
        int len = ptr - cursor;

        currentToken.setKind(Lpgsym.TK_SYMBOL);
        currentToken.setEndLocation(ptr - 1);
        String variableName = new String(inputBuffer, cursor, len);
        currentToken.setSymbol(variableTable.findOrInsert(
                variableName, new VariableSymbol(variableName, variableTable.size())));

        cursor = ptr;
    }

    private void classifyEquivalence()
    {
        if (inputBuffer[cursor + 1] == ':' &&
            inputBuffer[cursor + 2] == '=' &&
            Character.isWhitespace(inputBuffer[cursor + 3]))
        {
            currentToken.setKind(Lpgsym.TK_EQUIVALENCE);
            currentToken.setEndLocation(cursor + 2);
            cursor += 3;
        }
        else if (inputBuffer[cursor + 1] == ':' &&
                inputBuffer[cursor + 2] == '=' &&
                inputBuffer[cursor + 3] == '?' &&
                Character.isWhitespace(inputBuffer[cursor + 4]))
        {
            currentToken.setKind(Lpgsym.TK_PRIORITY_EQUIVALENCE);
            currentToken.setEndLocation(cursor + 3);
            cursor += 4;
        }
        else
            classifySymbol();
    }

    private void classifyArrow()
    {
        if (inputBuffer[cursor + 1] == '>' &&
            Character.isWhitespace(inputBuffer[cursor + 2]))
        {
            currentToken.setKind(Lpgsym.TK_ARROW);
            currentToken.setEndLocation(cursor + 1);
            cursor += 2;
        }
        else if (inputBuffer[cursor + 1] == '>' &&
                inputBuffer[cursor + 2] == '?' &&
                Character.isWhitespace(inputBuffer[cursor + 3]))
        {
            currentToken.setKind(Lpgsym.TK_PRIORITY_ARROW);
            currentToken.setEndLocation(cursor + 2);
            cursor += 3;
        }
        else
            classifySymbol();
    }

    private void classifyOr()
    {
        if (Character.isWhitespace(inputBuffer[cursor + 1]))
        {
            currentToken.setKind(Lpgsym.TK_OR_MARKER);
            currentToken.setEndLocation(cursor);
            cursor++;
        }
        else
            classifySymbol();
    }

    private void classifySingleQuotedSymbol()
    {
        char delimiter = inputBuffer[cursor];

        StringBuilder buf = new StringBuilder();
        char c;
        int ptr = cursor + 1;
        for (; (c = inputBuffer[ptr]) != '\r' && c != '\n'; ptr++)
        {
            if (c == delimiter)
            {
                if (inputBuffer[ptr + 1] != delimiter)
                    break;
                ptr++; // turn 2 consecutive quotes into one.
            }
            buf.append(c);
        }

        if (buf.length() == 0)
        {
            currentToken.setKind(Lpgsym.TK_EMPTY_KEY);
            currentToken.setEndLocation(ptr);
        }
        else
        {
            currentToken.resetInfoAndSetLocation(inputFile, cursor + 1);
            currentToken.setKind(Lpgsym.TK_SYMBOL);
            String variableName = new String(buf);
            currentToken.setSymbol(variableTable.findOrInsert(
                    variableName, new VariableSymbol(variableName, variableTable.size())));
            currentToken.setEndLocation(ptr - 1);
        }

        if (c == delimiter)
        {
            cursor = ptr + 1;
            //
            // TODO: This test appeared to have been here for legacy reasons.
            // Thus, it has been disabled until someone screams and then it will be
            // reevaluated.
            //
            // if (!Character.isWhitespace(inputBuffer[cursor]))
            //     addErrorToken(StreamErrorKind.UNDELIMITED_STRING_SYMBOL, currentTokenIndex);
        }
        else
        {
            cursor = ptr;
            addErrorToken(StreamErrorKind.UNTERMINATED_STRING_SYMBOL, currentTokenIndex);
        }
    }

    private void classifyDoubleQuotedSymbol()
    {
        char delimiter = inputBuffer[cursor];

        StringBuilder buf = new StringBuilder();
        char c;
        int ptr = cursor + 1;
        for (; (c = inputBuffer[ptr]) != '\r' && c != '\n'; ptr++)
        {
            if (c == delimiter)
            {
                if (inputBuffer[ptr + 1] != delimiter)
                    break;
                buf.append(c);
                ptr++; // turn 2 consecutive quotes into one.
            }
            else if (c == '\\')
            {
                switch (inputBuffer[ptr + 1])
                {
                    case 'b':
                        buf.append('\b');
                        ptr++;
                        break;
                    case 't':
                        buf.append('\t');
                        ptr++;
                        break;
                    case 'n':
                        buf.append('\n');
                        ptr++;
                        break;
                    case 'f':
                        buf.append('\f');
                        ptr++;
                        break;
                    case 'r':
                        buf.append('\r');
                        ptr++;
                        break;
                    case '"':
                        buf.append('"');
                        ptr++;
                        break;
                    case '\'':
                        buf.append('\'');
                        ptr++;
                        break;
                    case '\\':
                        buf.append('\\');
                        ptr++;
                        break;
                    case '0': case '1': case '2': case '3': case '4':
                    case '5': case '6': case '7': case '8': case '9':
                        {
                            int num = 0;
                            int i = 1;
                            for (; i < 4; i++)
                            {
                                int digit = Character.digit(inputBuffer[ptr + i], 8);
                                if (digit == -1)
                                    break;
                                num = num * 8 + digit;
                            }

                            //
                            // If the character is a ASCII code save it.
                            //
                            if (i > 1 && num <= 128)
                            {
                                buf.append((char)num);
                                ptr += i;
                            }
                            else
                            {
                                buf.append(c);
                                addErrorToken(StreamErrorKind.BAD_OCTAL_ASCII_CODE, currentTokenIndex);
                            }
                        }
                        break;
                    case 'u':
                        {
                            int num = 0;
                            int i = 2;
                            for (; i < 6; i++)
                            {
                                int digit = Character.digit(inputBuffer[ptr + i], 16);
                                if (digit == -1)
                                    break;
                                num = num * 16 + digit;
                            }

                            //
                            // If the character is a valid unicode character save it.
                            //
                            if (i == 6)
                            {
                                buf.append((char)num);
                                ptr += i;
                            }
                            else
                            {
                                buf.append(c);
                                addErrorToken(StreamErrorKind.BAD_UNICODE, currentTokenIndex);
                            }
                        }
                        break;
                    default:
                        buf.append(c);
                        addErrorToken(StreamErrorKind.ISOLATED_BACKSLASH, currentTokenIndex);
                        break;
                }
            }
            else
                buf.append(c);
        }

        if (buf.length() == 0)
        {
            currentToken.setKind(Lpgsym.TK_EMPTY_KEY);
            currentToken.setEndLocation(ptr);
        }
        else
        {
            currentToken.resetInfoAndSetLocation(inputFile, cursor + 1);
            currentToken.setKind(Lpgsym.TK_SYMBOL);
            String variableName = new String(buf);
            currentToken.setSymbol(variableTable.findOrInsert(
                    variableName, new VariableSymbol(variableName, variableTable.size())));
            currentToken.setEndLocation(ptr - 1);
        }

        if (c == delimiter)
        {
            cursor = ptr + 1;
            //
            // TODO: This test appeared to have been here for legacy reasons.
            // Thus, it has been disabled until someone screams and then it will be
            // reevaluated.
            //
            // if (!Character.isWhitespace(inputBuffer[cursor]))
            //     addErrorToken(StreamErrorKind.UNDELIMITED_STRING_SYMBOL, currentTokenIndex);
        }
        else
        {
            cursor = ptr;
            addErrorToken(StreamErrorKind.UNTERMINATED_STRING_SYMBOL, currentTokenIndex);
        }
    }

    private void classifyLess()
    {
        int ptr = cursor + 1;

        if (Character.isLetter(inputBuffer[ptr]))
        {
            char c;
            for (ptr++; (c = inputBuffer[ptr]) != '>' && c != '\r' && c != '\n'; ptr++)
                ;
            if (c == '>')
                ptr++;
            else
                addErrorToken(StreamErrorKind.UNTERMINATED_BRACKET_SYMBOL, currentTokenIndex);

            currentToken.setKind(Lpgsym.TK_SYMBOL);
            String variableName = new String(inputBuffer, cursor, ptr - cursor);
            currentToken.setSymbol(variableTable.findOrInsert(
                    variableName, new VariableSymbol(variableName, variableTable.size())));
            currentToken.setEndLocation(ptr - 1);
            cursor = ptr;
        }
        else
            classifySymbol();
    }

    private int processInclude(int start)
    {
        //
        // . Save position of INCLUDE keyword token
        // . Scan the include section
        // . Include the files specified in the include section
        //
        int includeKeyIndex = currentTokenIndex;
        int ptr = start;
        cursor = ptr;
        skipSpaces();
        if (cursor < inputEnd && inputBuffer[cursor] != option.escape)
        {
            classifyToken();
            skipSpaces();
            ptr = cursor;
            if (currentToken.kind() == Lpgsym.TK_SYMBOL)
            {
                String filename = lexStream.nameString(currentTokenIndex);

                Token saveIncludeKey = lexStream.getTokenReference(includeKeyIndex);
                Token saveFilename = currentToken;

                lexStream.resetTokenStream(includeKeyIndex);
                char c;
                if (cursor < inputEnd &&
                        ((c = inputBuffer[cursor]) == '%' || (c == option.escape && option.legacy)))
                {
                    int p = cursor + 1;
                    while (Character.isLetterOrDigit(c = inputBuffer[p]) && c != option.escape)
                        p++;
                    int len = p - cursor;
                    if (scanKeyword0(cursor, len) == Lpgsym.TK_END_KEY)
                        ptr = p; // move the pointer
                }

                pushEnvironment();

                inputFile = lexStream.findOrInsertFile(option.includeSearchDirectory, filename);
                StreamErrorKind errorCode = includeFile();

                if (errorCode != StreamErrorKind.NO_ERROR)
                {
                    currentTokenIndex = lexStream.getNextToken(inputFile, saveIncludeKey.startLocation());
                    lexStream.setToken(currentTokenIndex, saveIncludeKey);
                    currentToken = saveIncludeKey;

                    currentTokenIndex = lexStream.getNextToken(inputFile, saveFilename.startLocation());
                    lexStream.setToken(currentTokenIndex, saveFilename);
                    currentToken = saveFilename;

                    addErrorToken(errorCode, currentTokenIndex);

                    popEnvironment();
                    ptr = cursor;   //Phileas TODO: what does this mean?
                }
                else
                    popEnvironment();
            }
        }

        return ptr;
    }

    private StreamErrorKind includeFile()
    {
        inputBuffer = null;
        cursor = 0;

        //
        // If we could not open the include file, issue an error.
        //
        if (inputFile == null)
            return StreamErrorKind.NO_INCLUDE;

        //
        // If the file is already in the process of being included, issue an error and stop.
        //
        if (inputFile.isLocked())
            return StreamErrorKind.RECURSIVE_INCLUDE;

        //
        // Lock the include_file to avoid looping...
        //
        inputFile.lock();
        inputFile.readInput();
        inputBuffer = inputFile.buffer();
        if (inputBuffer == null)
            return StreamErrorKind.NO_INCLUDE;
        lineLocation = inputFile.lineLocation();
        lineLocation.clear();
        lineLocation.add(0); // mark starting location of line # 0
        skipOptions();
        scan(cursor);
        while (lexStream.kind(currentTokenIndex) == Lpgsym.TK_EOF)
            currentTokenIndex--;
        lexStream.resetTokenStream(currentTokenIndex + 1);
        inputFile.unlock();

        return StreamErrorKind.NO_ERROR;
    }

    private void pushEnvironment()
    {
        fileStack.add(inputFile);
        cursorStack.add(cursor);
    }

    private void popEnvironment()
    {
        assert (!fileStack.isEmpty());
        inputFile = fileStack.remove(fileStack.size() - 1);
        lineLocation = inputFile.lineLocation();
        inputBuffer = inputFile.buffer();
        inputEnd = inputBuffer.length;
        cursor = cursorStack.remove(cursorStack.size() - 1);
    }

    private void resetEnvironment(int cursor)
    {
        IntArrayList lineLocation = inputFile.lineLocation();
        int lineLocationSize = lineLocation.size();
        int length = lineLocationSize - 1;
        for (; length > 0; length--)
            if (lineLocation.get(length) <= cursor)
                break;
        lineLocation.removeRange(length + 1, lineLocationSize);
    }

    int numErrorTokens()
    {
        return errorTokens.size();
    }

    private void addErrorToken(StreamErrorKind errorKind, int index)
    {
        addErrorToken(errorKind, lexStream.nameString(index), lexStream.getTokenReference(index));
    }

    private void addErrorToken(StreamErrorKind errorKind, String name, Token token)
    {
        errorTokens.add(new ProblemToken(errorKind, name, token));
    }

    private void addWarningToken(StreamErrorKind warningKind, int index)
    {
        addWarningToken(warningKind, lexStream.nameString(index), lexStream.getTokenReference(index));
    }

    private void addWarningToken(StreamErrorKind warningKind, String name, Token token)
    {
        warningTokens.add(new ProblemToken(warningKind, name, token));
    }

    private final Option option;
    private final LexStream lexStream;
    private final LookupTable<String, VariableSymbol> variableTable;
    private final LookupTable<String, MacroSymbol> macroTable;
    private final Blocks actionBlocks;

    private InputFileSymbol inputFile = null;
    private IntArrayList lineLocation = null;
    private char[] inputBuffer = null;
    private int inputEnd = 0;
    private int cursor = 0;
    private Token currentToken = null;
    private int currentTokenIndex = 0;

    private List<InputFileSymbol> fileStack = new ArrayList<InputFileSymbol>();
    private IntArrayList cursorStack = new IntArrayList();

    private final List<ProblemToken> errorTokens = new ArrayList<ProblemToken>();
    private final List<ProblemToken> warningTokens = new ArrayList<ProblemToken>();

    private static class ProblemToken
    {
        private ProblemToken(StreamErrorKind msgCode, String name, Token token)
        {
            this.msgCode    = msgCode;
            this.name       = name;
            this.token      = token;
        }

        private final StreamErrorKind msgCode;
        private final String name;
        private final Token token;
    }

    private static enum StreamErrorKind
    {
        NO_ERROR,
        NO_INPUT,
        NO_TEMPLATE,
        NO_INCLUDE,
        LEGACY_KEYWORD,
        SYMBOL_WITH_KEYWORD_MARKER,
        RECURSIVE_INCLUDE,
        BAD_UNICODE,
        BAD_OCTAL_ASCII_CODE,
        ISOLATED_BACKSLASH,
        UNDELIMITED_STRING_SYMBOL,
        UNTERMINATED_STRING_SYMBOL,
        UNTERMINATED_BRACKET_SYMBOL,
        UNTERMINATED_BLOCK,
        INCLUDE_OPTIONS;
    }

}
